/*
 * To change this template, choose Tools | Templates
 * and open the template in the editor.
 */
package LM;

import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.OutputStreamWriter;
import java.io.UnsupportedEncodingException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.Map;
import java.util.StringTokenizer;

/**
 *
 * @author Administrator
 */
public class LetterLanguageModel {
    private HashMap<String, Integer> data = new HashMap<String, Integer>();
    private int total = 0;

    public static void main(String argv[]) throws FileNotFoundException, UnsupportedEncodingException, IOException {
       LetterLanguageModel llm=new LetterLanguageModel();
       llm.createModel("data\\llm\\awsd.db");
       llm.writeModelFile("data\\llm\\awsdLlm.db");
    }

    public HashMap<String, Integer> getData() {
        return data;
    }

    public int getTotal() {
        return total;
    }

    public HashMap<String,Integer> readInput(String fileName) throws FileNotFoundException, UnsupportedEncodingException, IOException {
//        BufferedReader br = new BufferedReader(new InputStreamReader(new FileInputStream(fileName), "UTF8"));
//        ArrayList inputData=new ArrayList();
//        String line;
//        while ((line = br.readLine()) != null) {
//            StringTokenizer st = new StringTokenizer(line);
//            Word w = new Word();
//            w.setStr(st.nextToken());
//            w.setCount(Integer.parseInt(st.nextToken()));
//            inputData.add(w);
//        }
//        return inputData;
        BufferedReader br = new BufferedReader(new InputStreamReader(new FileInputStream(fileName), "UTF8"));
        HashMap<String,Integer> inputData=new HashMap<String, Integer>();
        String line;
        while ((line = br.readLine()) != null) {
            StringTokenizer st = new StringTokenizer(line);
            String word=st.nextToken();
            int num=Integer.parseInt(st.nextToken());
            inputData.put(word, num);
        }
        return inputData;
    }
//Tu du danh sach cac tu tao ra model triagram
    public void createModel(String fileName) throws FileNotFoundException, UnsupportedEncodingException, IOException {
//        //get inputData
//        ArrayList inputData=readInput(fileName);
//        //set data
//        for (int i = 0; i < inputData.size(); i++) {
//            Word w = (Word) inputData.get(i);
//            String ws = w.getStr();
//            for (int j = 1; j <= ws.length(); j++) {
//                String letGroup = "";
//                if (j == 1) {
//                    letGroup = ws.substring(j - 1, j);
//                } else {
//                    if (j == 2) {
//                        letGroup = ws.substring(j - 2, j);
//                    } else {
//                        letGroup = ws.substring(j - 3, j);
//                    }
//
//                }
//                if(data.containsKey(letGroup)){
//                    int count=data.get(letGroup);
//                    count++;
//                    data.put(letGroup,count);
//                }else{
//                    
//                    data.put(letGroup, 1);
//                }
//            }
//        }
//        //set total
//        for(Map.Entry<String,Integer>e :data.entrySet()){
//            total+=e.getValue();
//        }
         //get inputData

        System.out.println("package LM; class LetterLanguageModel");

        HashMap<String,Integer> inputData=readInput(fileName);
        //set data
        for (Map.Entry<String,Integer> entry: inputData.entrySet()) {
            String word = entry.getKey();
            for (int j = 1; j <= word.length(); j++) {
                String letGroup = "";
                if (j == 1) {
                    letGroup = word.substring(0, 1);
                } else {
                    if (j == 2) {
                        letGroup = word.substring(0, 2);
                    } else {
                        letGroup = word.substring(j - 3, j);
                    }

                }
                if(data.containsKey(letGroup)){
                    int count=data.get(letGroup);
                    count=count+entry.getValue();
                    data.put(letGroup,count);
                }else{
                    int num=entry.getValue();
                    data.put(letGroup, num);
                }
            }
        }
        //set total
        for(Map.Entry<String,Integer>e :data.entrySet()){
            total+=e.getValue();
        }
    }

    public double getEntropy(String word) {
        double entropy = 1;
        for (int i = 1; i <= word.length(); i++) {
            String letGroup = "";
            if (i == 1) {
                letGroup = word.substring(i - 1, i);

            } else {
                if (i == 2) {
                    letGroup = word.substring(i - 2, i);
                } else {
                    letGroup = word.substring(i - 3, i);
                }
            }
          if(data.containsKey(letGroup)){
              entropy=entropy*data.get(letGroup)/total;
          }else{
                entropy=entropy*0.0;
          }

        }
        return entropy;
    }
 //ghi model ra file
    public void writeModelFile(String fileName) throws FileNotFoundException, UnsupportedEncodingException, IOException {
        BufferedWriter bw=new BufferedWriter(new OutputStreamWriter(new FileOutputStream(fileName),"UTF8"));
      
        ArrayList dataList = new ArrayList(data.entrySet());
        Collections.sort(dataList, new Comparator() {

            public int compare(Object o1, Object o2) {
                Map.Entry e1 = (Map.Entry) o1;
                Map.Entry e2 = (Map.Entry) o2;
                Integer first = (Integer) e1.getValue();
                Integer second = (Integer) e2.getValue();
                return second.compareTo(first);
            }
        });
        for (int i = 0; i < dataList.size(); i++) {
            Map.Entry e = (Map.Entry)dataList.get(i);
            String line = e.getKey() + " " + e.getValue() + "\n";
            bw.write(line);
        }
        bw.close();

    }
    
    //loadmodel
    public void loadModel(String filename) throws FileNotFoundException, UnsupportedEncodingException, IOException {
       BufferedReader br=new BufferedReader(new InputStreamReader(new FileInputStream(filename),"UTF8"));
       String line;
       total=0;
       while((line=br.readLine())!=null){
           StringTokenizer st=new StringTokenizer(line);
           String str=st.nextToken();
           int num= Integer.parseInt(st.nextToken());
           data.put(str,num);
           total+=num;
       }
    }
}
